# Updated by Wei 2013/07/27 night

# program purpose: 
# (1) For the previous step, in order to make the computation fast, I distributed the job into 10 pieces for the feature generation like:
# postingRankInDoc and postingRankInList
# (2) So the previous program will output me two training files. One with the postingRankInListAdded and one with the postingRankInDocAdded
# (3) This program is for me to combine those two training files into ONE for further processing

from __future__ import division
from operator import itemgetter, attrgetter
import os
import sys
import math

# inputFileName1 = "/data3/obukai/the_new_trip_of_feature_generation/gov2ClearYourMindAndDoItAgain/MLRelated/Model15/TrainingSetForFixingTheMissingTOPKResults20130715_tfqAdded_tfcAdded_postingRankInDocAdded_WITHOUT_HEADLINE_sorted_by_trecID.train"
inputFileName1 = "/data3/obukai/the_new_trip_of_feature_generation/gov2ClearYourMindAndDoItAgain/MLRelated/Model17/TOPK_sorted_by_queryID/distributedWorkForFeatureGenerationPostingRankInDoc20130727/TrainingSetForTOPKMissingResults20130719_tfqAdded_tfcAdded_postingRankInDocAdded_WITHOUT_HEADLINE_sorted_by_queryID.train"
# inputFileName2 = "/data3/obukai/the_new_trip_of_feature_generation/gov2ClearYourMindAndDoItAgain/MLRelated/Model15/TrainingSetForFixingTheMissingTOPKResults20130715_tfqAdded_tfcAdded_WITHOUT_HEADLINE_postingRankInListAdded_sorted_by_trecID.train"
inputFileName2 = "/data3/obukai/the_new_trip_of_feature_generation/gov2ClearYourMindAndDoItAgain/MLRelated/Model17/TOPK_sorted_by_queryID/distributedWorkForFeatureGenerationPostingRankInList20130727/TrainingSetForTOPKMissingResults20130719_tfqAdded_tfcAdded_postingRankInListAdded_WITHOUT_HEADLINE_sorted_by_queryID.train"
# outputFileName = "/data3/obukai/the_new_trip_of_feature_generation/gov2ClearYourMindAndDoItAgain/MLRelated/Model15/TrainingSetForFixingTheMissingTOPKResults20130715_tfqAdded_tfcAdded_postingRankInDocAdd_postingRankInListAdd_WITH_headline.train"
outputFileName = "/data3/obukai/the_new_trip_of_feature_generation/gov2ClearYourMindAndDoItAgain/MLRelated/Model17/TrainingSetForTOPKMissingResults20130719_tfqAdded_tfcAdded_postingRankInDocAdd_postingRankInListAdd_WITH_headline.train"
inputFileHandler1 = open(inputFileName1,"r")
inputFileHandler2 = open(inputFileName2,"r")
outputFileHandler = open(outputFileName,"w")

# write the headline
# example training instance line: 2 1 GX271-17-12723767 25074378 orange 1.948974 0.747498 1.456854 283475 1 1627 13.39897 3 117 600634
outputFileHandler.write("SelectedRankIndex queryID externalTrecID internalDocID term partialBM25ScoreComponentPart1_IDF partialBM25ScoreComponentPart2_TF partialBM25 length_of_inverted_index term_freq_in_doc doc_words overallBM25Score rank_in_result_list term_freq_in_training_head95K_queries term_freq_in_collection postingRankInDoc postingRankInList" + "\n")

currentLineFromFile1 = inputFileHandler1.readline()
currentLineFromFile2 = inputFileHandler2.readline()

while currentLineFromFile1 and currentLineFromFile2:
    currentLineElementsFromFile1 = currentLineFromFile1.strip().split(" ")
    currentLineElementsFromFile2 = currentLineFromFile2.strip().split(" ")
    
    # the same training instance, let's add the features to it
    theSameFlag = True
    for i in range(0,len(currentLineElementsFromFile1)-1):
        if currentLineElementsFromFile1[i] != currentLineElementsFromFile2[i]:
            print "currentLineElementsFromFile1[i]:",currentLineElementsFromFile1[i]
            print "currentLineElementsFromFile2[i]:",currentLineElementsFromFile2[i]
            theSameFlag = False
    if not theSameFlag:
        print "system error"
        exit(1)
    else:
        outputFileHandler.write(currentLineFromFile1.strip() + " " + currentLineElementsFromFile2[-1] + "\n")
    
    currentLineFromFile1 = inputFileHandler1.readline()
    currentLineFromFile2 = inputFileHandler2.readline()
    
inputFileHandler1.close()
inputFileHandler2.close()
outputFileHandler.close()
print "Overall Processing Statistics:"
print "inputFileName1:",inputFileName1
print "inputFileName2:",inputFileName2
print "outputFileName:",outputFileName
print "Done"